# This function goes to a sitemap page, and extract all the urls found
extract_url_from_sitemap <- function(url, delay = 1) { 
  urls <- read_html(url) %>% 
    html_elements(xpath = ".//loc") %>% 
    html_text()
  Sys.sleep(delay) # You set a delay to avoid overloading the website
  return(urls)
}

# insistently allows to retry when you did not succeed in loading the page
insistently_extract_url <- insistently(extract_url_from_sitemap, rate = rate_backoff(max_times = 5)) 

document_pages <- extract_url_from_sitemap(session$robotstxt$sitemap$value) %>% 
  .[str_detect(., "documents")] # We keep only the URLs for documents

bis_pages <- map(document_pages[1:5], # showing the code just on the first five years
                 ~insistently_extract_url(url = ., 
                                          delay = session$delay))

bis_pages <- tibble(year = str_extract(document_pages[1:5], "\\d{4}"),
                    urls = bis_pages) %>% 
  unnest(urls)

data_page <- tibble(date = remote_driver$findElements("css selector", ".item_date") %>% 
                      map_chr(., ~.$getElementText()[[1]]),
                    info = remote_driver$findElements("css selector", ".item_date+ td") %>% 
                      map_chr(., ~.$getElementText()[[1]]),
                    url = remote_driver$findElements("css selector", ".dark") %>% 
                      map_chr(., ~.$getElementAttribute("href")[[1]])) %>% 
  separate(info, c("title", "description", "speaker"), "\n")